package downloader.bswx9

import org.slf4j.LoggerFactory
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.Selectable

class Bxwx9PageProcesser : PageProcessor {
    val Log = LoggerFactory.getLogger(javaClass)
    val urls = mutableMapOf<String,Int>()
    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(100).setCycleRetryTimes(3).setTimeOut(30_000)
    }

    override fun process(page: Page) {
        val html = page.html
        val mainInfo: Selectable = html.xpath("//div[@id='TabCss']")

        if (mainInfo.match()) {
            val dd = mainInfo.`$`("a").nodes()
            dd.forEach{
                urls.put(it.links().get(),it.xpath("/a/text()").get().toInt())
            }
            page.addTargetRequests(urls.keys.toList())

        } else {
            val title = html.xpath("//div[@id='title']/text()").get()
            page.putField("title", title)
            var content = html.xpath("//div[@id='content']/text()").get().replace(Regex("\\s+"),"\r\n").replace("\u00a0","")
            page.putField("content", content)
            page.putField("idx",urls.getValue(page.url.get()))

        }
    }
}